source /mnt/afs2d/luotianhang/basic_set/stop_make_url.sh
export DS_SKIP_CUDA_CHECK=1
export RESUME_NAME='/mnt/afs2d/luotianhang/smartvehicle_diffusion/diffusers/examples/inpainting/sdxl_inpainting_training4/checkpoint-14000'
export MODEL_NAME='/mnt/afs2d/luotianhang/cache/PretrainedModels/models--diffusers--stable-diffusion-xl-1.0-inpainting-0.1/snapshots/115134f363124c53c7d878647567d04daf26e41e'
accelerate launch --config_file=./deepspeed_config.yaml   train_sdxl.py \
    --pretrained_model_name_or_path=$MODEL_NAME \
    --resolution=128  \
    --train_batch_size=2 --gradient_accumulation_steps=1 --gradient_checkpointing \
    --max_train_steps=250000 \
    --checkpointing_steps=500 \
    --checkpoints_total_limit=2 \
    --max_grad_norm=1 --lr_warmup_steps=0 \
    --mixed_precision=fp16 \
    --seed=1204 \
    --num_workers=2 \
    --output_dir=sdxl_inpainting_training4 \
    --use_8bit_adam \
    --variant=fp16 \
    --resume_from_checkpoint=$RESUME_NAME 